{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Residual networks with PyTorch\n",
    "\n",
    "In this example, we'll implement the various types of residual blocks using PyTorch. We'll train the network on the CIFAR-10 dataset.\n",
    "\n",
    "_The code in this section is partially based on the pre-activation ResNet implementation in_ [https://github.com/kuangliu/pytorch-cifar](https://github.com/kuangliu/pytorch-cifar)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's start with the imports:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torchvision\n",
    "from torchvision import transforms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll continue by implementing the pre-activation non-bottleneck residual block, which is a subclass of `torch.nn.Module`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PreActivationBlock(nn.Module):\n",
    "    \"\"\"Pre-activation residual block.\"\"\"\n",
    "\n",
    "    expansion = 1\n",
    "\n",
    "    def __init__(self, in_slices, slices, stride=1):\n",
    "        super(PreActivationBlock, self).__init__()\n",
    "\n",
    "        self.bn_1 = nn.BatchNorm2d(in_slices)\n",
    "        self.conv_1 = nn.Conv2d(in_channels=in_slices, out_channels=slices,\n",
    "                                kernel_size=3, stride=stride, padding=1,\n",
    "                                bias=False)\n",
    "\n",
    "        self.bn_2 = nn.BatchNorm2d(slices)\n",
    "        self.conv_2 = nn.Conv2d(in_channels=slices, out_channels=slices,\n",
    "                                kernel_size=3, stride=1, padding=1,\n",
    "                                bias=False)\n",
    "\n",
    "        # if the input/output dimensions differ use convolution for the shortcut\n",
    "        if stride != 1 or in_slices != self.expansion * slices:\n",
    "            self.shortcut = nn.Sequential(\n",
    "                nn.Conv2d(in_channels=in_slices,\n",
    "                          out_channels=self.expansion * slices,\n",
    "                          kernel_size=1,\n",
    "                          stride=stride,\n",
    "                          bias=False)\n",
    "            )\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = F.relu(self.bn_1(x))\n",
    "\n",
    "        #  reuse bn+relu in down-sampling layers\n",
    "        shortcut = self.shortcut(out) if hasattr(self, 'shortcut') else x\n",
    "\n",
    "        out = self.conv_1(out)\n",
    "\n",
    "        out = F.relu(self.bn_2(out))\n",
    "        out = self.conv_2(out)\n",
    "\n",
    "        out += shortcut\n",
    "\n",
    "        return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll continue with the implementation of the pre-activation bottleneck residual block:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PreActivationBottleneckBlock(nn.Module):\n",
    "    \"\"\"Pre-activation bottleneck residual block.\"\"\"\n",
    "\n",
    "    expansion = 4\n",
    "\n",
    "    def __init__(self, in_slices, slices, stride=1):\n",
    "        super(PreActivationBottleneckBlock, self).__init__()\n",
    "\n",
    "        self.bn_1 = nn.BatchNorm2d(in_slices)\n",
    "        self.conv_1 = nn.Conv2d(in_channels=in_slices, out_channels=slices,\n",
    "                                kernel_size=1,\n",
    "                                bias=False)\n",
    "\n",
    "        self.bn_2 = nn.BatchNorm2d(slices)\n",
    "        self.conv_2 = nn.Conv2d(in_channels=slices, out_channels=slices,\n",
    "                                kernel_size=3, stride=stride, padding=1,\n",
    "                                bias=False)\n",
    "\n",
    "        self.bn_3 = nn.BatchNorm2d(slices)\n",
    "        self.conv_3 = nn.Conv2d(in_channels=slices,\n",
    "                                out_channels=self.expansion * slices,\n",
    "                                kernel_size=1,\n",
    "                                bias=False)\n",
    "\n",
    "        # if the input/output dimensions differ use convolution for the shortcut\n",
    "        if stride != 1 or in_slices != self.expansion * slices:\n",
    "            self.shortcut = nn.Sequential(\n",
    "                nn.Conv2d(in_channels=in_slices,\n",
    "                          out_channels=self.expansion * slices,\n",
    "                          kernel_size=1, stride=stride,\n",
    "                          bias=False)\n",
    "            )\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = F.relu(self.bn_1(x))\n",
    "\n",
    "        #  reuse bn+relu in down-sampling layers\n",
    "        shortcut = self.shortcut(out) if hasattr(self, 'shortcut') else x\n",
    "\n",
    "        out = self.conv_1(out)\n",
    "\n",
    "        out = F.relu(self.bn_2(out))\n",
    "        out = self.conv_2(out)\n",
    "\n",
    "        out = F.relu(self.bn_3(out))\n",
    "        out = self.conv_3(out)\n",
    "\n",
    "        out += shortcut\n",
    "\n",
    "        return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, we'll implement a configurable ResNet class, which works with both types of pre-activation blocks. The network takes as input the type of pre-activation block, the number of residual groups, the number of residual blocks in each group, and the number of output classes (10 in the case of CIFAR-10). The class builds the network according to these parameters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PreActivationResNet(nn.Module):\n",
    "    \"\"\"Pre-activation residual network\"\"\"\n",
    "\n",
    "    def __init__(self, block, num_blocks, num_classes=10):\n",
    "        \"\"\"\n",
    "        :param block: type of residual block (regular or bottleneck)\n",
    "        :param num_blocks: a list with 4 integer values.\n",
    "            Each value reflects the number of residual blocks in the group\n",
    "        :param num_classes: number of output classes\n",
    "        \"\"\"\n",
    "\n",
    "        super(PreActivationResNet, self).__init__()\n",
    "\n",
    "        self.in_slices = 64\n",
    "\n",
    "        self.conv_1 = nn.Conv2d(in_channels=3, out_channels=64,\n",
    "                                kernel_size=3, stride=1, padding=1,\n",
    "                                bias=False)\n",
    "\n",
    "        self.layer_1 = self._make_group(block, 64, num_blocks[0], stride=1)\n",
    "        self.layer_2 = self._make_group(block, 128, num_blocks[1], stride=2)\n",
    "        self.layer_3 = self._make_group(block, 256, num_blocks[2], stride=2)\n",
    "        self.layer_4 = self._make_group(block, 512, num_blocks[3], stride=2)\n",
    "        self.linear = nn.Linear(512 * block.expansion, num_classes)\n",
    "\n",
    "    def _make_group(self, block, slices, num_blocks, stride):\n",
    "        \"\"\"Create one residual group\"\"\"\n",
    "\n",
    "        strides = [stride] + [1] * (num_blocks - 1)\n",
    "        layers = []\n",
    "        for stride in strides:\n",
    "            layers.append(block(self.in_slices, slices, stride))\n",
    "            self.in_slices = slices * block.expansion\n",
    "\n",
    "        return nn.Sequential(*layers)\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = self.conv_1(x)\n",
    "        out = self.layer_1(out)\n",
    "        out = self.layer_2(out)\n",
    "        out = self.layer_3(out)\n",
    "        out = self.layer_4(out)\n",
    "        out = F.avg_pool2d(out, 4)\n",
    "        out = out.view(out.size(0), -1)\n",
    "        out = self.linear(out)\n",
    "\n",
    "        return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, we'll define several `PreActivationResNet` configurations, which match the configuration of the original ResNet papers:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def PreActivationResNet18():\n",
    "    return PreActivationResNet(block=PreActivationBlock,\n",
    "                               num_blocks=[2, 2, 2, 2])\n",
    "\n",
    "\n",
    "def PreActivationResNet34():\n",
    "    return PreActivationResNet(block=PreActivationBlock,\n",
    "                               num_blocks=[3, 4, 6, 3])\n",
    "\n",
    "\n",
    "def PreActivationResNet50():\n",
    "    return PreActivationResNet(block=PreActivationBottleneckBlock,\n",
    "                               num_blocks=[3, 4, 6, 3])\n",
    "\n",
    "\n",
    "def PreActivationResNet101():\n",
    "    return PreActivationResNet(block=PreActivationBottleneckBlock,\n",
    "                               num_blocks=[3, 4, 23, 3])\n",
    "\n",
    "\n",
    "def PreActivationResNet152():\n",
    "    return PreActivationResNet(block=PreActivationBottleneckBlock,\n",
    "                               num_blocks=[3, 8, 36, 3])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll continue with the implementation of the model training routine, which is generic and is not specifically related to the `PreActivationResNet`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, loss_function, optimizer, data_loader):\n",
    "    \"\"\"Train one epoch\"\"\"\n",
    "\n",
    "    # set model to training mode\n",
    "    model.train()\n",
    "\n",
    "    current_loss = 0.0\n",
    "    current_acc = 0\n",
    "\n",
    "    # iterate over the training data\n",
    "    for i, (inputs, labels) in enumerate(data_loader):\n",
    "        # send the input/labels to the GPU\n",
    "        inputs = inputs.to(device)\n",
    "        labels = labels.to(device)\n",
    "\n",
    "        # zero the parameter gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        with torch.set_grad_enabled(True):\n",
    "            # forward\n",
    "            outputs = model(inputs)\n",
    "            _, predictions = torch.max(outputs, 1)\n",
    "            loss = loss_function(outputs, labels)\n",
    "\n",
    "            # backward\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "        # statistics\n",
    "        current_loss += loss.item() * inputs.size(0)\n",
    "        current_acc += torch.sum(predictions == labels.data)\n",
    "\n",
    "    total_loss = current_loss / len(data_loader.dataset)\n",
    "    total_acc = current_acc.double() / len(data_loader.dataset)\n",
    "\n",
    "    print('Train Loss: {:.4f}; Accuracy: {:.4f}'.format(total_loss, total_acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then, we'll implement the validation routine:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_model(model, loss_function, data_loader):\n",
    "    \"\"\"Test for a single epoch\"\"\"\n",
    "\n",
    "    # set model in evaluation mode\n",
    "    model.eval()\n",
    "\n",
    "    current_loss = 0.0\n",
    "    current_acc = 0\n",
    "\n",
    "    # iterate over  the validation data\n",
    "    for i, (inputs, labels) in enumerate(data_loader):\n",
    "        # send the input/labels to the GPU\n",
    "        inputs = inputs.to(device)\n",
    "        labels = labels.to(device)\n",
    "\n",
    "        # forward\n",
    "        with torch.set_grad_enabled(False):\n",
    "            outputs = model(inputs)\n",
    "            _, predictions = torch.max(outputs, 1)\n",
    "            loss = loss_function(outputs, labels)\n",
    "\n",
    "        # statistics\n",
    "        current_loss += loss.item() * inputs.size(0)\n",
    "        current_acc += torch.sum(predictions == labels.data)\n",
    "\n",
    "    total_loss = current_loss / len(data_loader.dataset)\n",
    "    total_acc = current_acc.double() / len(data_loader.dataset)\n",
    "\n",
    "    print('Test Loss: {:.4f}; Accuracy: {:.4f}'.format(total_loss, total_acc))\n",
    "\n",
    "    return total_loss, total_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll continue with the `plot_accuracy` function (the name speaks for itself):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_accuracy(accuracy: list):\n",
    "    \"\"\"Plot accuracy\"\"\"\n",
    "    plt.figure()\n",
    "    plt.plot(accuracy)\n",
    "    plt.xticks(\n",
    "        [i for i in range(0, len(accuracy))],\n",
    "        [i + 1 for i in range(0, len(accuracy))])\n",
    "    plt.ylabel('Accuracy')\n",
    "    plt.xlabel('Epoch')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now have all the ingredients to build, train, and validate the model. Let's start by instantiating the training dataset transformations (`transform_train`) and `train_loader`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "# training data transformation\n",
    "transform_train = transforms.Compose([\n",
    "    transforms.RandomCrop(32, padding=4),\n",
    "    transforms.RandomHorizontalFlip(),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.4914, 0.4821, 0.4465), (0.2470, 0.2435, 0.2616))\n",
    "])\n",
    "\n",
    "# training data loader\n",
    "train_set = torchvision.datasets.CIFAR10(root='./data',\n",
    "                                         train=True,\n",
    "                                         download=True,\n",
    "                                         transform=transform_train)\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_set,\n",
    "                                           batch_size=100,\n",
    "                                           shuffle=True,\n",
    "                                           num_workers=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's do the same for the validation dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "# test data transformation\n",
    "transform_test = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.4914, 0.4821, 0.4465), (0.2470, 0.2435, 0.2616))\n",
    "])\n",
    "\n",
    "# test data loader\n",
    "testset = torchvision.datasets.CIFAR10(root='./data',\n",
    "                                       train=False,\n",
    "                                       download=True,\n",
    "                                       transform=transform_test)\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(dataset=testset,\n",
    "                                          batch_size=100,\n",
    "                                          shuffle=False,\n",
    "                                          num_workers=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll continue by instantiating the `device`, `model`, `loss_function`, and the `optimizer`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the model\n",
    "model = PreActivationResNet34()\n",
    "\n",
    "# select gpu 0, if available\n",
    "# otherwise fallback to cpu\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# transfer the model to the GPU\n",
    "model = model.to(device)\n",
    "\n",
    "# loss function\n",
    "loss_function = nn.CrossEntropyLoss()\n",
    "\n",
    "# We'll optimize all parameters\n",
    "optimizer = optim.Adam(model.parameters())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, we'll run the training for 15 epochs and we'll plot the accuracy at the end:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/15\n",
      "Train Loss: 1.5397; Accuracy: 0.4284\n",
      "Test Loss: 1.3968; Accuracy: 0.4944\n",
      "Epoch 2/15\n",
      "Train Loss: 1.2436; Accuracy: 0.5450\n",
      "Test Loss: 1.1301; Accuracy: 0.6010\n",
      "Epoch 3/15\n",
      "Train Loss: 1.0482; Accuracy: 0.6239\n",
      "Test Loss: 0.9587; Accuracy: 0.6745\n",
      "Epoch 4/15\n",
      "Train Loss: 0.8672; Accuracy: 0.6948\n",
      "Test Loss: 0.8985; Accuracy: 0.6996\n",
      "Epoch 5/15\n",
      "Train Loss: 0.7243; Accuracy: 0.7480\n",
      "Test Loss: 0.6420; Accuracy: 0.7819\n",
      "Epoch 6/15\n",
      "Train Loss: 0.6335; Accuracy: 0.7836\n",
      "Test Loss: 0.6720; Accuracy: 0.7736\n",
      "Epoch 7/15\n",
      "Train Loss: 0.5695; Accuracy: 0.8058\n",
      "Test Loss: 0.5958; Accuracy: 0.8042\n",
      "Epoch 8/15\n",
      "Train Loss: 0.5033; Accuracy: 0.8262\n",
      "Test Loss: 0.5246; Accuracy: 0.8223\n",
      "Epoch 9/15\n",
      "Train Loss: 0.4667; Accuracy: 0.8429\n",
      "Test Loss: 0.5277; Accuracy: 0.8344\n",
      "Epoch 10/15\n",
      "Train Loss: 0.4256; Accuracy: 0.8554\n",
      "Test Loss: 0.4896; Accuracy: 0.8371\n",
      "Epoch 11/15\n",
      "Train Loss: 0.3892; Accuracy: 0.8671\n",
      "Test Loss: 0.4600; Accuracy: 0.8609\n",
      "Epoch 12/15\n",
      "Train Loss: 0.3638; Accuracy: 0.8741\n",
      "Test Loss: 0.4029; Accuracy: 0.8686\n",
      "Epoch 13/15\n",
      "Train Loss: 0.3441; Accuracy: 0.8846\n",
      "Test Loss: 0.4014; Accuracy: 0.8701\n",
      "Epoch 14/15\n",
      "Train Loss: 0.3166; Accuracy: 0.8930\n",
      "Test Loss: 0.4746; Accuracy: 0.8446\n",
      "Epoch 15/15\n",
      "Train Loss: 0.3003; Accuracy: 0.8980\n",
      "Test Loss: 0.6309; Accuracy: 0.8365\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZzWdbn/8dfF7GzDjjADDMi+CMSEC265ohmYpWF1XNI4dtLMTov+SistT3U8x6U8ncxIT2VIVoZKogjmrgwKwrANss0MywzbMCzDbNfvj/s7dDvewCz3d+5Z3s/H437Md73ua7bvdX++y+dj7o6IiEh9nRKdgIiItE4qECIiEpMKhIiIxKQCISIiMalAiIhITMmJTiBe+vTp4zk5OYlOQ0SkTVm2bNkud+8ba127KRA5OTnk5eUlOg0RkTbFzLYca51OMYmISEwqECIiEpMKhIiIxKQCISIiMalAiIhITCoQIiISkwqEiIjE1G6egxARqc/dOVJdy+HKGiqqa6ioip6ue9VSUVXD4ahpgCs+lsWAzIwEfweJpQIhIm2Gu7O9rILV2/azevt+1u7Yz56DlUcP7EcP+NU1HK6s4Uh1bZPf63+WbOAbF43i2tOHkJzUMU+2qECISKtUWV3LhpIDrN6+nzXb9x8tCmWHq45uk9O7M/26pdMtPZm+3dLISEkiPaUT6SlJZKQkkRbMR5b/czotJYn05CQyUoPt66aTk0hL6UTJ/iPcNX8V9zy7mj8vK+LeKyYwaVCPBP40EsPay4hyubm5rq42RNqmskNVrN4eKQCrt0UKQkFJOVU1keNTekonRp3UnbEDujN2YHfGDujGqJO60zUtvM+47s7zq3bwg2fyKSk/whdOHcy3Lh5NZkZKaO+ZCGa2zN1zY61TC0JEWkxtrVO09zCrt5cFLYJy1mzfT/G+w0e36dstjbEDunPOqL6MGRApCkP7dCGpk7VormbGJRMGcOaIPtz/YgGPvbGJ51ft5M7LxjBj4kDMWjafRFALQkRCdaS6ht+9uYWF+TtYu72c8iPVAHQyOLlv10gRGBgpBGMGdKdvt7QEZxzbquIyvvvXlawoKmPa8N7cM3M8w/p2TXRazXa8FoQKhIiEwt1ZtKaEHz23mi27DzExO5OJg3ocLQSjTupGekpSotNslJpa54l3tvKz59dypKqWr5x7Ml859+Q2931EU4EQkRa1bkc59zy7mtc27GJ4v67cedlYzhkZc8iBNqmkvIIfP7eGvy3fRk7vztxz+XjOGtE2vz8VCBFpEXsPVnL/ovX84e2tdElN4rYLR/LF04aQ0k5vE32tYBd3/m0Vm3YdZMbEgXzvsjH065ae6LQaJWEFwsymAw8CScCj7v6TeusHA48DPYJtbnf3BWaWA6wB1gWbvuXuNx3vvVQgRBKnqqaWP7y1hfsXFVBeUcUXTxvCbReMpGeX1ESnFrqKqhr+9x8f8D9LPiAtpRPfvngUnz91SItfVG+qhBQIM0sC1gMXAkXAUuBqd18dtc0jwHvu/kszGwsscPecoEA86+7jG/p+KhAiifHK+lLueXY1BSUHmDa8N3deNpbRJ3VPdFotbmPpAe76Wz6vbdjFxOxMfvzpCYzPykx0Wid0vAIRZrtvKrDB3Te6eyUwF5hZbxsH6v6SMoFtIeYjInG0addBbnx8KdfMeYfKmloe+Zcp/P6GUztkcQAY1rcrv7thKg/OmkTxvgpm/OI1fvhMPuUVVSfeuZUK8zmILKAwar4IOLXeNj8AXjCzW4AuwAVR64aa2XvAfuB77v5q/Tcws9nAbIDBgwfHL3MROab9FVX8YvEGfvv6JlKTOnH7JaO5floOaclt906eeDEzZk7K4txR/bhv4Toee2MzC1Zu5/ufGscl409qc89OhHmK6bPAdHe/MZj/F+BUd785aptvBDn8l5mdDvwGGA+kAF3dfbeZTQGeBsa5+/5jvZ9OMYmEq6bW+VNeIfe9sI7dByu5cko237x4VJu7KNuSlhfu47t/XUn+tv2cM7IvP5wxjpw+XRKd1ock6knqYmBQ1Hx2sCzaDcB0AHd/08zSgT7uXgIcCZYvM7MPgJGAKoBIAry9cTc/fGY1q7fvJ3dIT3573VQmZLf+8+uJNmlQD/721Wn835tb+K8X1nHufS+T3TOD8QMzmZCdybiB3RmflUmfrq3z4cAwC8RSYISZDSVSGGYBn6+3zVbgfOAxMxsDpAOlZtYX2OPuNWY2DBgBbAwxVxGJoWjvIf5jwVqeW7mdgZnp/PzqyVx2yoA2d6okkZKTOvGlM4dy6YQBPL28mJXFZeQXl/F8/o6j25zUPZ3xWZFiMX5gJuOzMunfPS3hP+fQCoS7V5vZzcBCIrewznH3fDO7G8hz9/nAvwO/NrPbiFywvs7d3czOBu42syqgFrjJ3feElauIfNihymr+9+UP+NUrGzGD2y4Yyeyzh5GRqusMTXVSZjo3nXPy0fn9FVWs3rafVcVl5G/bz8riMl5aW0LdWf8+XVOjCkZ3xg3MJLtnRosWDT0oJyJApM+kssNVvL5hFz/9+zp27K9g5qSBfGf6aAb26NgD57SUQ5XVrNm+n1XFkcKxsriMgpID1NRGjtM9OqcwfmAm47K6H21pDOnVmU7NeOZCvbmKdADuzsHKGvYfrqIseH1ouqL6OOuqqKj65+A6p2Rn8vAXJjNlSK8EfkcdT+fUZKYM6fWhn3tFVQ3rdpSzalsZq4r3k7+tjN++tpnKmsjvq1taMp8Y3Y+Hrp4c93xUIETamE27DrJ4bQlvfrCL0vIjHzr4V9ce+4yAWeRg0j0jhczgNbxfV7qnp5DZOTLfPT2ZrJ4ZnDuyX7M+lUr8pKckMXFQDyZGDVhUWV1LQUk5+cX7WbWtLLRxMVQgRFq5I9U1vLNpD0vWlrJkXQmbdh0EYFjfLmT37Mzg3l3IzEiOHOiDA390EYgc+FPomp7cZrp/kONLTe7EuIGZjBuYyVUfulk0vlQgRFqhnfsrWLK2hMVrS3h9wy4OVtaQmtyJM07uzfXTcvjEqH4M6tU50WlKO6cCIdIK1NQ6K4r2HS0K+dsiz4QOzEzn8slZnDe6H2ec3Ed3EUmLUoEQSZCyw1W8sr6UJWtLeHl9KXsOVtLJYMqQnnx7+ijOG92PUf27JfxeeOm4VCBEWoi7U1BygMVBK2HZlr3U1Do9Oqdw7si+fGJ0P84Z2Zcendt/F9nSNqhAiITI3Xnjg908v2oHi9eWULzvMABjBnTnpnOGcd7ofkwa1FMXj6VVUoEQCcHhyhr++l4xv319EwUlB8hISWLa8D589RPD+cTovgzI1INn0vqpQIjE0Y6yCv7vzc088c5W9h2qYtzA7vz3VRO5dMKANj2wvXRMKhAicbCicB9zXt/Ec+9vp8adi8b254Yzh/HxnJ66yCxtlgqESBNV19SyMH8nc17fxLIte+malsy1Z+Rw7ek5DO6tZxSk7VOBEGmkskNVzF26lcff2My2sgoG9+rM9z81ls9OyaZbekqi0xOJGxUIkQbaWHqAx97YzFPLijhUWcNpw3rxgxnjOH9Mf92FJO2SCoTIcbg7r2/YzZzXN7F4bQmpSZ2YMWkg10/LYdxAjagm7ZsKhEgMFVU1PP1eMXNe38T6nQfo0zWVr18wgi+cOoS+3Vrn8JAi8aYCIRJl5/4Kfv/WFv7w9lb2HKxkzIDu3HflRD41cQBpybpNVTqWUAuEmU0HHiQy5Oij7v6TeusHA48DPYJtbnf3BcG6O4AbgBrga+6+MMxcpWM7Ul3DT/++jt+9tZnqWueCMf350rShnDasl25TlQ4rtAJhZknAw8CFQBGw1Mzmu/vqqM2+B8xz91+a2VhgAZATTM8CxgEDgUVmNtLda8LKVzquTbsOcssf32VV8X6unjqIm845mSG9uyQ6LZGEC7MFMRXY4O4bAcxsLjATiC4QDnQPpjOBbcH0TGCuux8BNpnZhiDemyHmKx3Q35YX8//+spLkpE78+ppcLhzbP9EpibQaYRaILKAwar4IOLXeNj8AXjCzW4AuwAVR+75Vb9+s+m9gZrOB2QCDBw+OS9LSMRyurOGHz+Qzd2khU4b05KGrJ5PVQ/0jiUTrlOD3vxp4zN2zgUuB35lZg3Ny90fcPdfdc/v27RtaktK+rN9ZzsyHX+PJvEL+7dyTmTv7NBUHkRjCbEEUw4cGS80OlkW7AZgO4O5vmlk60KeB+4o0irszL6+Q78/Pp2taMo9fP5WzR+qDhcixhNmCWAqMMLOhZpZK5KLz/HrbbAXOBzCzMUA6UBpsN8vM0sxsKDACeCfEXKWdO3Ckmq8/uZzv/HklHxvckwW3nqXiIHICobUg3L3azG4GFhK5hXWOu+eb2d1AnrvPB/4d+LWZ3UbkgvV17u5AvpnNI3JBuxr4qu5gkqZaVVzGzU+8y9Y9h/j3C0fyb58Yrq4xRBrAIsfjti83N9fz8vISnYa0Iu7O429s5t4Fa+nVJZUHZ03i1GG9E52WSKtiZsvcPTfWOj1JLe1S2aEqvvXUCl5YvZPzRvfjvisn0quLxnoWaQwVCGl3lm3Zy9f++B4791fwvU+O4UvThtJJp5REGk0FQkJ1qLKa9OSkFjlA19Y6j7y6kf9cuI6BPdJ56itnMGlQj9DfV6S9UoGQ0HxQeoBLH3yV1KROjM/KZEJ2JhOyIq8hvTvHtY+jXQeO8I15K3hlfSmXTjiJ/7jiFDIzNHiPSHOoQEhofv5SAZ3MmDFpIKuKy3js9c1U1tQC0C09+WixqCscg3s1rWi8+cFubp37HvsOV3HP5eP54qmD1cGeSByoQEgoNpQcYP6KbXz5rGHccekYACqra1m/s5xVxWW8X1zGquIyfhtVNDIzUhif1Z3xWZmcktWDCVmZDOqVccyDfU2t89BLBfx8cQE5vbvw2PVTGTuwe8xtRaTxVCAkFD9fXEBachKzzx52dFlqcuRU0/isTGYFy+qKxvtFZawMisac1zZRVRO5/TozI4UJwT6nBC2N7J4ZlJQf4da57/HWxj1cMTmLey4fT5c0/TmLxJP+oyTuNpSUM3/FNmafPYzeXY8/+lp00ahzpLqG9TsO8H7xvkhro6iMR1/dSHVtpGj06JxCba1TVePcd+VEPjslO9TvR6SjUoGQuHvopQ1kpCQx+6xhJ944hrTkpMh1iex/Fo2KqhrW7ShnZXEZK4vKKDtcxTcvHsnwft3ilbaI1KMCIXFVsLOcZ97fxr+effIJWw+NkZ6SxMRBPZio21ZFWkyiu/uWduahxUHr4eymtR5EpPVQgZC4Wb+znGff38a1Z+SoWwuRdkAFQuLmoZcK6JySxJebeO1BRFoXFQiJi/U7y3lu5Xa1HkTaERUIiYsHXyqgS2qyWg8i7YgKhDTbuh3lLFi5nevOyKGnWg8i7YYKhDTbQ0Hr4cazhiY6FRGJo1ALhJlNN7N1ZrbBzG6Psf5+M1sevNab2b6odTVR6+qPZS2txNod+3lu5Xaun5ZDj85qPYi0J6E9KGdmScDDwIVAEbDUzOa7++q6bdz9tqjtbwEmR4U47O6TwspP4uPBRQV0S0vmhjPVehBpb8JsQUwFNrj7RnevBOYCM4+z/dXAH0PMR+Jszfb9/H3VDrUeRNqpMAtEFlAYNV8ULPsIMxsCDAUWRy1ON7M8M3vLzC4/xn6zg23ySktL45W3NNA/Ww+6c0mkPWotF6lnAU+5e03UsiHungt8HnjAzE6uv5O7P+Luue6e27dv35bKVYDV2/bzfP4Orj9zKJmdNXKbSHsUZoEoBgZFzWcHy2KZRb3TS+5eHHzdCLzMh69PSII9+NJ6uqUnc8M0XXsQaa/CLBBLgRFmNtTMUokUgY/cjWRmo4GewJtRy3qaWVow3QeYBqyuv68kRv62Mhbm7+RL09R6EGnPQruLyd2rzexmYCGQBMxx93wzuxvIc/e6YjELmOvuHrX7GOBXZlZLpIj9JPruJ0msBxcV0C09mS/pziWRdi3U8SDcfQGwoN6yu+rN/yDGfm8AE8LMTZpmVXEZL6zeydcvGEFmhloPIu1Za7lILW3Egy9FWg/X69qDSLunAiENtqq4jBdX7+TGM4ep9SDSAahASIM9sKiA7unJXH9mTqJTEZEWoAIhDbKquIxFa3Zy41nD6J6u1oNIR6ACIQ3ywKL1dE9P5rppOYlORURaiAqEnNDKojIWrSnhy2o9iHQoKhByQg8sWk9mRopaDyIdjAqEHNf7Rft4aW0JXz5rKN3UehDpUFQg5LgeWFRAj84pXHtGTqJTEZEWpgIhx7SicB+L10auPaj1INLxnLBAmNktZtazJZKR1uWBRevVehDpwBrSguhPZLjQecEY0xZ2UpJ4ywv3sWRdKV8+axhd00LtsktEWqkTFgh3/x4wAvgNcB1QYGb3xhrAR9qPBxatp6daDyIdWoOuQQRdce8IXtVExm94ysx+FmJukiDvbd3Ly+tK+fLZaj2IdGQn/O83s1uBa4BdwKPAt9y9ysw6AQXAt8NNUVraA4sKIq2H03MSnYqIJFBDPh72Aq5w9y3RC9291swuCyctSZR3t+7lH+tL+c700XRR60GkQ2vIKaa/A3vqZsysu5mdCuDua8JKTBLjgUUF9OqSyjWnD0l0KiKSYA0pEL8EDkTNHwiWSTuzbMteXllfyuyzh6n1ICINKhAWPV60u9fSwKFKg9ti15nZBjO7Pcb6+81sefBab2b7otZda2YFwevahryfNM8Di9ar9SAiRzXkQL/RzL7GP1sN/wZsPNFOZpYEPAxcCBQReZZivruvrtvG3W+L2v4WYHIw3Qv4PpALOLAs2Hdvg74rabRlW/bwasEu7rhkNJ1T1XoQkYa1IG4CzgCKiRzoTwVmN2C/qcAGd9/o7pXAXGDmcba/GvhjMH0x8KK77wmKwovA9Aa8pzRB4Z5D/PCZ1fTuksq/qPUgIoETflR09xJgVhNiZwGFUfN1xeUjzGwIMBRYfJx9s2LsN5ugWA0ePLgJKXZsR6pr+PUrG/nFkg10MuNnnz1FrQcROaohz0GkAzcA44D0uuXu/qU45jELeMrdaxqzk7s/AjwCkJub6yfYXKK8VrCLu/62io27DnLJ+JO487KxDOyRkei0RKQVacgppt8BJxE57fMPIBsob8B+xcCgqPnsYFkss/jn6aXG7iuNsKOsgpufeJcv/uZtatx57PqP88svTlFxEJGPaMj5hOHufqWZzXT3x83sCeDVBuy3FBhhZkOJHNxnAZ+vv5GZjSbSdcebUYsXAvdG9SJ7EXBHA95TjqG6ppbH3tjM/S+up6rWue2CkfzrOcNIT0lKdGoi0ko1pEBUBV/3mdl4Iv0x9TvRTu5ebWY3EznYJwFz3D3fzO4G8tx9frDpLGBuvVtp95jZPUSKDMDd7r4HaZKlm/dw59OrWLujnE+M6ssPZoxjSO8uiU5LRFo5izoux97A7Ebgz8AE4DGgK3Cnu/8q9OwaITc31/Py8hKdRquy+8AR/uPva3lqWREDM9O561PjuHhcf9Rju4jUMbNl7p4ba91xWxBBh3z7g1tNXwGGhZCfxFlNrfPHd7bynwvXcfBINTedczJfO3+47lASkUY57hEj6JDv28C8FspHmmllURnfe3olK4rKOG1YL+6ZOZ4R/bslOi0RaYMa8pFykZl9E3gSOFi3UNcEWpeyQ1Xc98I6fv/2Fnp3SePBWZOYMXGgTieJSJM1pEB8Lvj61ahljk43tQruzl/eLebeBWvYe6iSa0/P4RsXjaR7ekqiUxORNq4hT1IPbYlEpPHW7SjnzqdX8c7mPUwe3IPHvzSV8VmZiU5LRNqJhjxJfU2s5e7+f/FPRxriwJFqHly0njmvb6ZbejI/uWICV+UOolMnnU4SkfhpyCmmj0dNpwPnA+8CKhAJ8PbG3dw6dzk79lcw6+OD+Pb00fTqkprotESkHWrIKaZboufNrAeRnlmlhVVU1fCNeStIS+nEn79yBlOG9DzxTiIiTdSUG+MPEul5VVrYnNc3UbzvME/ceKqKg4iEriHXIJ4hctcSRDr3G4uei2hxpeVH+J8lH3DBmP6cMbxPotMRkQ6gIS2I+6Kmq4Et7l4UUj5yDPcvWk9FVQ13XDo60amISAfRkAKxFdju7hUAZpZhZjnuvjnUzOSodTvKmfvOVq45PYeT+3ZNdDoi0kE0ZDyIPwG1UfM1wTJpIT9esIauacncev6IRKciIh1IQwpEcjCmNADBtO6rbCEvryvhlfWlfO38EfTU7awi0oIaUiBKzWxG3YyZzQR2hZeS1KmuqeXeBWvI6d2Za07PSXQ6ItLBNOQaxE3AH8zsF8F8ERDz6WqJryfzClm/8wD/+8UppCY3pJaLiMRPQx6U+wA4zcy6BvMHQs9KKK+o4r9fWM/Uob24eFz/RKcjIh3QCT+Wmtm9ZtbD3Q+4+wEz62lmP2pIcDObbmbrzGyDmd1+jG2uMrPVZpYfjHddt7zGzJYHr/mx9m3P/uflD9h9sJI7PzlWXXaLSEI05LzFJe6+r24mGF3u0hPtZGZJwMPAJUQerrvazMbW22YEcAcwzd3HAV+PWn3Y3ScFrxl0IIV7DvGb1zZxxeQsJmSrd1YRSYyGFIgkM0urmzGzDCDtONvXmQpscPeNwZ1Pc4GZ9bb5MvBwUHRw95KGpd2+/WzhOjoZfPPiUYlORUQ6sIYUiD8AL5nZDWZ2I/Ai8HgD9ssCCqPmi4Jl0UYCI83sdTN7y8ymR61LN7O8YPnlsd7AzGYH2+SVlpY2IKXWb9mWvTyzYhuzzxrGwB4ZiU5HRDqwhlyk/qmZrQAuINIn00JgSBzffwRwLpANvGJmE4JTWkPcvdjMhgGLzWxlcME8OrdHgEcAcnNznTbO3fnRc6vp2y2Nfz3n5ESnIyIdXEPvndxJpDhcCZwHrGnAPsXAoKj57GBZtCJgvrtXufsmYD2RgoG7FwdfNwIvA5MbmGub9ez723lv6z6+ddEouqQ1paNdEZH4OWaBMLORZvZ9M1sL/JxIn0zm7p9w918ca78oS4ERZjbUzFKBWUD9u5GeJtJ6wMz6EDnltDG4Uyotavk0YHXjvrW2paKqhp/8fS1jBnTnM1OyE52OiMhxTzGtBV4FLnP3DQBmdltDA7t7tZndTOSUVBIwx93zzexuIM/d5wfrLjKz1UT6ePqWu+82szOAX5lZLZEi9hN3b9cF4revb6Z432F+9tlTSNLQoSLSChyvQFxB5FP/EjN7nshdSI06crn7AmBBvWV3RU078I3gFb3NG8CExrxXW7brwBEeXrKBC8b0Y5rGehCRVuKYp5jc/Wl3nwWMBpYQeUahn5n90swuaqkEO4L7X6wb62FMolMRETnqhBep3f2guz/h7p8icqH5PeA7oWfWQazfWc4f39nKF08borEeRKRVaVQPcO6+190fcffzw0qoo7lXYz2ISCulLkIT6B/rS3l5XSm3nKexHkSk9VGBSJDqmlp+/NxqBvfqzDVnxOu5QxGR+FGBSJB5eUWs33mAOy4ZTVpyUqLTERH5CBWIBCivqOK/X1zHx3N6Mn38SYlOR0QkJhWIBPjlyx+w60Al39NYDyLSiqlAtLCivYd49LVNfHpyFhMH9Uh0OiIix6QC0cJ+9vw6DPiWxnoQkVZOBaIFvbt1L/NXbGP22RrrQURaPxWIFuLu/OjZyFgPN2msBxFpA1QgWshzK7fz7tZ9fPOikRrrQUTaBBWIFlA31sPok7rx2SmDTryDiEgroALRAh57YzNFew/zvU+O1VgPItJmqECEbPeBIzy8eAPnje7HmSM01oOItB0qECF7YFEBh6pq+H+Xjk50KiIijRJqgTCz6Wa2zsw2mNntx9jmKjNbbWb5ZvZE1PJrzawgeF0bZp5hKdhZzhPvbOULpw5meL9uiU5HRKRRQrudxsySgIeBC4EiYKmZzY8eW9rMRgB3ANPcfa+Z9QuW9wK+D+QCDiwL9t0bVr5huHfBGjqnJmmsBxFpk8JsQUwFNrj7RnevJDKm9cx623wZeLjuwO/uJcHyi4EX3X1PsO5FYHqIucbdWxt3s2RdKbecN5zeXdMSnY6ISKOFWSCygMKo+aJgWbSRwEgze93M3jKz6Y3YFzObbWZ5ZpZXWloax9Sb7/dvbaFH5xSuOT0n0amIiDRJoi9SJwMjgHOBq4Ffm1mDe7ALhj/Ndffcvn37hpRi4+09WMkL+Tu5fFIW6Ska60FE2qYwC0QxEP1UWHawLFoRMN/dq9x9E7CeSMFoyL6t1l/fK6ayppbPfVwPxYlI2xVmgVgKjDCzoWaWCswC5tfb5mkirQfMrA+RU04bgYXARWbW08x6AhcFy1o9d2deXiETszMZM6B7otMREWmy0AqEu1cDNxM5sK8B5rl7vpndbWYzgs0WArvNbDWwBPiWu+929z3APUSKzFLg7mBZq/d+URlrd5RzlVoPItLGhdprnLsvABbUW3ZX1LQD3whe9fedA8wJM78wPJlXSHpKJz41cWCiUxERaZZEX6RuVw5VVjN/+TYunTCA7ukpiU5HRKRZVCDiaMHKHRw4Us2sjw9OdCoiIs2mAhFH85YWMqxPFz6e0zPRqYiINJsKRJxsLD3AO5v3cGXuIMzUpbeItH0qEHHyZF4hSZ2Mz0z5yAPfIiJtkgpEHFTV1PLnZcWcN7of/bqlJzodEZG4UIGIgyVrS9h14Aify9WzDyLSfqhAxMG8vEL6dUvj3FGtpz8oEZHmUoFopp37K1i8toTPTMkmOUk/ThFpP3REa6anlhVR63CVTi+JSDujAtEM7s6f8go5dWgvhvbpkuh0RETiSgWiGd7etIfNuw+pW28RaZdUIJrhyaWFdEtL5pLxAxKdiohI3KlANFHZ4SoWrNzOjEkDyUjVqHEi0v6oQDTR/BXbOFJdq475RKTdUoFoonlLCxkzoDvjszRqnIi0TyoQTZC/rYyVxWV8LjdbHfOJSLulAtEE85YWkprcicsnq2M+EWm/Qi0QZjbdzNaZ2QYzuz3G+uvMrNTMlgevG6PW1UQtnx9mno1RUVXD08u3cfG4k+jROTXR6YiIhCa0ManNLAl4GLgQKAKWmtl8d19db9Mn3f3mGCEOu9mdrSQAAAwISURBVPuksPJrqoX5Oyg7XMUsPfsgIu1cmC2IqcAGd9/o7pXAXGBmiO/XIublFTKoVwanD+ud6FREREIVZoHIAgqj5ouCZfV9xszeN7OnzCz6Y3m6meWZ2VtmdnmsNzCz2cE2eaWlpXFMPbatuw/x+obdXDllEJ066eK0iLRvib5I/QyQ4+6nAC8Cj0etG+LuucDngQfM7OT6O7v7I+6e6+65ffuG39X2n5YVYgafnZId+nuJiCRamAWiGIhuEWQHy45y993ufiSYfRSYErWuOPi6EXgZmBxiridUU+s8tayIc0b2ZWCPjESmIiLSIsIsEEuBEWY21MxSgVnAh+5GMrPoToxmAGuC5T3NLC2Y7gNMA+pf3G5RrxSUsr2sQqPGiUiHEdpdTO5ebWY3AwuBJGCOu+eb2d1AnrvPB75mZjOAamAPcF2w+xjgV2ZWS6SI/STG3U8t6sl3CundJZXzx/RPZBoiIi0mtAIB4O4LgAX1lt0VNX0HcEeM/d4AJoSZW2PsOnCERWt2ct0ZOaQmJ/qyjYhIy9DRrgH++m4x1bWucR9EpENRgTgBd+fJvEI+NrgHI/p3S3Q6IiItRgXiBN7duo8NJQfUehCRDkcF4gSeXLqVzqlJfPKUgYlORUSkRalAHMeBI9U8+/52LjtlAF3TQr2eLyLS6qhAHMdz72/jUGUNn9OocSLSAalAHMeTSwsZ3q8rHxvcI9GpiIi0OBWIYyjYWc67W/fxudxBGjVORDokFYhjeHJpIcmdjE9/TKPGiUjHpAIRQ2V1LX95r5gLx/anT9e0RKcjIpIQKhAxvLRmJ3sOVnKVnn0QkQ5MBSKGuUsLGZCZztkjwh9jQkSktVKBqGfbvsO8UlDKZ6dkk6RR40SkA1OBqOepZUW4w5VTdHpJRDo2FYgotbXOvLxCpg3vzeDenROdjohIQqlARHnjg90U7T3MVRo1TkREBSLak3mFZGakcPG4kxKdiohIwoVaIMxsupmtM7MNZnZ7jPXXmVmpmS0PXjdGrbvWzAqC17Vh5gmw71AlC/N3cPmkgaSnJIX9diIirV5oXZSaWRLwMHAhUAQsNbP5McaWftLdb663by/g+0Au4MCyYN+9YeX79HvFVFbXqmM+EZFAmC2IqcAGd9/o7pXAXGBmA/e9GHjR3fcEReFFYHpIeeLuzF1ayISsTMYO7B7W24iItClhFogsoDBqvihYVt9nzOx9M3vKzOquDjdoXzObbWZ5ZpZXWlra5ERXFpexdke5npwWEYmS6IvUzwA57n4KkVbC443Z2d0fcfdcd8/t27fpTz0/ubSQtOROzJioUeNEROqEWSCKgeiP5NnBsqPcfbe7HwlmHwWmNHTfeDlcWcP85dv45IQBZGakhPEWIiJtUpgFYikwwsyGmlkqMAuYH72BmQ2Imp0BrAmmFwIXmVlPM+sJXBQsi7vyiirOHd2PWVN1cVpEJFpodzG5e7WZ3UzkwJ4EzHH3fDO7G8hz9/nA18xsBlAN7AGuC/bdY2b3ECkyAHe7+54w8uzXPZ2fXz05jNAiIm2auXuic4iL3Nxcz8vLS3QaIiJtipktc/fcWOsSfZFaRERaKRUIERGJSQVCRERiUoEQEZGYVCBERCQmFQgREYlJBUJERGJqN89BmFkpsKUZIfoAu+KUTpgxFTe8mIobXkzFDS9mc+MOcfeYndm1mwLRXGaWd6yHRVpTTMUNL6bihhdTccOLGWZcnWISEZGYVCBERCQmFYh/eqSNxFTc8GIqbngxFTe8mKHF1TUIERGJSS0IERGJSQVCRERi6vAFwszmmFmJma2KY8xBZrbEzFabWb6Z3RqnuOlm9o6ZrQji/jAecYPYSWb2npk9G8eYm81spZktN7O4DdZhZj3M7CkzW2tma8zs9DjEHBXkWffab2Zfj0Pc24Lf1Soz+6OZpTc3ZhD31iBmfnPyjPX3b2a9zOxFMysIvvaMU9wrg3xrzaxJt2QeI+5/Bn8L75vZX82sRxxi3hPEW25mL5hZowesP96xxcz+3czczPrEI66Z/cDMiqP+fi9tbNyY3L1Dv4CzgY8Bq+IYcwDwsWC6G7AeGBuHuAZ0DaZTgLeB0+KU8zeAJ4Bn4/hz2Az0CeF39jhwYzCdCvSIc/wkYAeRB4iaEycL2ARkBPPzgOvikN94YBXQmciokIuA4U2M9ZG/f+BnwO3B9O3AT+MUdwwwCngZyI1jvhcBycH0Txub7zFido+a/hrwv/HINVg+iMhIm1ua8v9xjHx/AHyzuX9b9V8dvgXh7q8QGe40njG3u/u7wXQ5kbG2s+IQ1939QDCbEryafZeBmWUDnwQebW6ssJlZJpF/kN8AuHulu++L89ucD3zg7s15Mr9OMpBhZslEDujb4hBzDPC2ux9y92rgH8AVTQl0jL//mUSKMMHXy+MR193XuPu6puR5grgvBD8HgLeA7DjE3B8124Um/J8d59hyP/DtpsQ8Qdy46/AFImxmlgNMJvJpPx7xksxsOVACvOju8Yj7AJE/2No4xIrmwAtmtszMZscp5lCgFPhtcErsUTPrEqfYdWYBf2xuEHcvBu4DtgLbgTJ3f6G5cYm0Hs4ys95m1hm4lMin0njp7+7bg+kdQP84xg7bl4C/xyOQmf3YzAqBLwB3xSnmTKDY3VfEI149NwenxeY05bRgLCoQITKzrsCfga/X+0TSZO5e4+6TiHxKmmpm45uZ42VAibsvi0d+9Zzp7h8DLgG+amZnxyFmMpHm9S/dfTJwkMhpkLgws1RgBvCnOMTqSeTT+FBgINDFzL7Y3LjuvobIqZQXgOeB5UBNc+Me472cOLRSW4KZfReoBv4Qj3ju/l13HxTEu7m58YJi/v+IU7Gp55fAycAkIh9G/iseQVUgQmJmKUSKwx/c/S/xjh+cVlkCTG9mqGnADDPbDMwFzjOz3zczJnD0EzTuXgL8FZgah7BFQFFUy+kpIgUjXi4B3nX3nXGIdQGwyd1L3b0K+AtwRhzi4u6/cfcp7n42sJfIda542WlmAwCCryVxjB0KM7sOuAz4QlDU4ukPwGfiEOdkIh8WVgT/b9nAu2Z2UnMDu/vO4MNjLfBr4vO/pgIRBjMzIufI17j7f8cxbt+6OzTMLAO4EFjbnJjufoe7Z7t7DpFTK4vdvdmfcs2si5l1q5smciGx2XeKufsOoNDMRgWLzgdWNzdulKuJw+mlwFbgNDPrHPxNnE/kelSzmVm/4OtgItcfnohH3MB84Npg+lrgb3GMHXdmNp3IKdIZ7n4oTjFHRM3OpJn/ZwDuvtLd+7l7TvD/VkTkZpYdzY1dV9ADnyYO/2uA7mIicjDYDlQR+YXdEIeYZxJplr9PpPm/HLg0DnFPAd4L4q4C7orzz+Jc4nQXEzAMWBG88oHvxjHPSUBe8HN4GugZp7hdgN1AZhxz/SGRg8sq4HdAWpzivkqkMK4Azm9GnI/8/QO9gZeAAiJ3SPWKU9xPB9NHgJ3AwjjF3QAURv2vNeqOo2PE/HPwO3sfeAbIikeu9dZvpml3McXK93fAyiDf+cCAePydqasNERGJSaeYREQkJhUIERGJSQVCRERiUoEQEZGYVCBERCQmFQiRRjCzmnq9vsbzKe6cWD1/iiRKcqITEGljDnukqxORdk8tCJE4sMjYFz+zyPgX75jZ8GB5jpktDjpReyl48hkz6x+MXbAieNV1wZFkZr8Oxk14IXhiXiQhVCBEGiej3immz0WtK3P3CcAviPSQC/Bz4HF3P4VInz4PBcsfAv7h7hOJ9CWVHywfATzs7uOAfcSnDyCRJtGT1CKNYGYH3L1rjOWbgfPcfWPQUeMOd+9tZruIdHtQFSzf7u59zKwUyHb3I1Excoh04T4imP8OkOLuPwr/OxP5KLUgROLHjzHdGEeipmvQdUJJIBUIkfj5XNTXN4PpN4j0kguRgWdeDaZfAr4CRweBymypJEUaSp9ORBonIxjRr87z7l53q2tPM3ufSCvg6mDZLURGv/sWkZHwrg+W3wo8YmY3EGkpfIVID50irYauQYjEQXANItfddyU6F5F40SkmERGJSS0IERGJSS0IERGJSQVCRERiUoEQEZGYVCBERCQmFQgREYnp/wNwR7+1uFvbLQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "EPOCHS = 15\n",
    "\n",
    "test_acc = list()  # collect accuracy for plotting\n",
    "for epoch in range(EPOCHS):\n",
    "    print('Epoch {}/{}'.format(epoch + 1, EPOCHS))\n",
    "\n",
    "    train_model(model, loss_function, optimizer, train_loader)\n",
    "    _, acc = test_model(model, loss_function, test_loader)\n",
    "    test_acc.append(acc)\n",
    "\n",
    "plot_accuracy(test_acc)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
